package com.quan.junit.area;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.alibaba.fastjson.JSON;
import com.quan.common.utils.HttpClientUtil;

/**
 * 从国家统计局获取城市信息
 * “市辖区”不作为节点
 * @author yhaoquan
 *
 */
public class SpiderAreaSimple {
	//(<a href='[\d]*.html'>)[\w\W]*</a>
	//<a.*?/a>
	//^(\d{6})(\d{4})(\d{2})(\d{2})(\d{3})([0-9]|X)$
	//<a http://\d.[a-zA-Z]+/?[/.?[/S|/s]]+[a>]$
	
	//一级
	String ProvinceUrl = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/index.html" ;
	Pattern p1 = Pattern.compile("<a.*?/a>") ; //匹配A标签
	Pattern p2 = Pattern.compile("[\\d]..html") ;//匹配URL地址
	Pattern p3 = Pattern.compile(">.*?</a>") ;//匹配A标签内容
	
	//二级
	String level2 = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/" ;
	Pattern p22 = Pattern.compile("([^'][\\d][/][\\d]*.html)") ;//<a href='11/1101.html'>110100000000</a>
	
	//三级
	String level3 = "http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2017/" ;
	
	
	/**
	 * 爬取官网数据
	 * @return
	 * @throws FileNotFoundException
	 */
	public List<AreaModel> getSpiderArea() throws FileNotFoundException {
		File file = new File("d:/area.txt") ;
		File file2 = new File("d:/area.json") ;
		PrintStream ps = new PrintStream(new FileOutputStream(file));
		PrintStream ps2 = new PrintStream(new FileOutputStream(file2));
		int index = 0 ;
		
		List<AreaModel> list = new ArrayList<AreaModel>() ;
		
		List<Map<String, String>> province = getProvince() ;
		for (Map<String, String> map : province) {
			System.out.println(map);
			ps.println(map.get("text"));
			
			AreaModel areaModel = new AreaModel() ;
			areaModel.setId(Integer.parseInt(map.get("code").substring(0, 6)));
			areaModel.setPid(0);
			areaModel.setLevel(1);
			areaModel.setCode(map.get("code"));
			areaModel.setName(map.get("text"));
			areaModel.setLeaf(false);
			list.add(areaModel) ;
			
			//二级
			List<AreaModel> child2 = new ArrayList<AreaModel>() ;
			List<Map<String, String>> level2 = getLevel2(map) ;
			for (Map<String, String> map2 : level2) {
				System.out.println(map2);
				ps.println("\t"+map2.get("text"));
				
				if("市辖区".equals(map2.get("text"))) {
					
					int level2Code = Integer.parseInt(map2.get("code").substring(0, 6)) ;
					
					int childId = 1 ;
					//三级
					List<AreaModel> child3 = new ArrayList<AreaModel>() ;
					List<Map<String, String>> level3 = getLevel2(map2) ;
					for (Map<String, String> map3 : level3) {
						System.out.println(map3);
						ps.println("\t\t"+map3.get("text"));
						
						if(!"市辖区".equals(map3.get("text"))) {
							AreaModel areaModel3 = new AreaModel() ;
							areaModel3.setId(level2Code+childId);
							areaModel3.setPid(areaModel.getId());
							areaModel3.setLevel(2);
							areaModel3.setCode(map3.get("code"));
							areaModel3.setName(map3.get("text"));
							areaModel3.setLeaf(true);
							child3.add(areaModel3) ;
							
							areaModel.setChildren(child3);
							
							childId++;
						}
						
					}
				} else {
					AreaModel areaModel2 = new AreaModel() ;
					areaModel2.setId(Integer.parseInt(map2.get("code").substring(0, 6)));
					areaModel2.setPid(areaModel.getId());
					areaModel2.setLevel(2);
					areaModel2.setCode(map2.get("code"));
					areaModel2.setName(map2.get("text"));
					areaModel2.setLeaf(false);
					child2.add(areaModel2) ;
					
					areaModel.setChildren(child2);
					
					int childId = 1 ;
					//三级
					List<AreaModel> child3 = new ArrayList<AreaModel>() ;
					List<Map<String, String>> level3 = getLevel2(map2) ;
					for (Map<String, String> map3 : level3) {
						System.out.println(map3);
						ps.println("\t\t"+map3.get("text"));
						
						if(!"市辖区".equals(map3.get("text"))) {
							AreaModel areaModel3 = new AreaModel() ;
							areaModel3.setId(areaModel2.getId()+childId);
							areaModel3.setPid(areaModel2.getId());
							areaModel3.setLevel(3);
							areaModel3.setCode(map3.get("code"));
							areaModel3.setName(map3.get("text"));
							areaModel3.setLeaf(true);
							child3.add(areaModel3) ;
							
							areaModel2.setChildren(child3);
							
							childId++;
						}
						
					}
				}
				
				
			}
			
			index++;
			if(index == 1) {
				break ;
			}
			
		}
		
		ps.close();
		
		ps2.println(JSON.toJSONString(list));
		ps2.close();
		
		System.out.println(JSON.toJSONString(list));
		
		return list ;
	}
	
	/**
	 * 迭代所有，用于插入数据库
	 * @param list
	 * @return
	 */
	public List<AreaModel> selectAll(List<AreaModel> list) {
		
		List<AreaModel> allList = new ArrayList<AreaModel>() ;
		
		//一级
		for (AreaModel area : list) {
			System.out.println(area.getId() +"  "+ area.getPid() +"  "+ area.getLevel() +"  "+ area.getName());
			allList.add(area) ;
			
			//二级
			if(null != area.getChildren() && area.getChildren().size() > 0) {
				for(AreaModel area2 : area.getChildren()) {
					System.out.println("\t" + area2.getId() +"  "+ area2.getPid() +"  "+ area2.getLevel() +"  "+  area2.getName());
					allList.add(area2) ;
					
					//三级
					if(null != area2.getChildren() && area2.getChildren().size() > 0) {
						for(AreaModel area3 : area2.getChildren()) {
							System.out.println("\t\t" + area3.getId() +"  "+ area3.getPid() +"  "+ area3.getLevel() +"  "+  area3.getName());
							allList.add(area3) ;
						}
					}
					
				}
			}
		}
		
		return allList ;
	}
	
	//二级
	public List<Map<String, String>> getLevel2(Map<String, String> mapObject) {
		String url = level2 + mapObject.get("nextUrl") ;
		String string = HttpClientUtil.getInstance().get(url, "GBK") ;
		
		
		int i=1;
		String code = "" ;
		
		List<Map<String, String>> list = new ArrayList<Map<String, String>>() ;
		
		Matcher m1 = p1.matcher(string) ;
		while(m1.find()) {
			String a = m1.group() ;
			//System.out.println(a);
			
			String nextUrl = "" ;
			String text = "" ;
			Matcher m2 = p22.matcher(a) ;
			while(m2.find()) {
				nextUrl = m2.group() ;
			}
			
			Matcher m3 = p3.matcher(a) ;
			while(m3.find()) {
				text = m3.group().replaceAll(">|</a>","") ;
			}
			
			if(i%2==0) {
				Map<String, String> map = new HashMap<String, String>() ;
				map.put("nextUrl", nextUrl) ;
				map.put("code", code) ;
				map.put("text", text) ;
				list.add(map);
			} else {
				code = text ;
			}
			i++;
		}
		
		return list ;
	}
	
	//一级
	public List<Map<String, String>> getProvince() {
		String string = HttpClientUtil.getInstance().get(ProvinceUrl, "GBK") ;
		
		List<Map<String, String>> list = new ArrayList<Map<String, String>>() ;
		
		Matcher m1 = p1.matcher(string) ;
		while(m1.find()) {
			String a = m1.group().replaceAll("<br/>", "") ;
			
			String nextUrl = "" ;
			String code = "" ;
			String text = "" ;
			
			Matcher m2 = p2.matcher(a) ;
			while(m2.find()) {
				nextUrl = m2.group() ;
			}
			
			Matcher m3 = p3.matcher(a) ;
			while(m3.find()) {
				text = m3.group().replaceAll(">|</a>","") ;
			}
			
			Matcher m4 = Pattern.compile("\\d.").matcher(a); 
			while(m4.find()) {
				code = m4.group()+"0000000000" ;
			}
			
			Map<String, String> map = new HashMap<String, String>() ;
			map.put("nextUrl", nextUrl) ;
			map.put("code", code) ;
			map.put("text", text) ;
			list.add(map);
			
			//System.out.println(nextUrl + "   " + code + "   " + text);
		}
		
		return list ;
	}

}
